{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 1,
   "metadata": {
    "scrolled": true
   },
   "outputs": [
    {
     "name": "stdout",
     "output_type": "stream",
     "text": [
      "Extracting /home/jhon/data/train-images-idx3-ubyte.gz\n",
      "Extracting /home/jhon/data/train-labels-idx1-ubyte.gz\n",
      "Extracting /home/jhon/data/t10k-images-idx3-ubyte.gz\n",
      "Extracting /home/jhon/data/t10k-labels-idx1-ubyte.gz\n",
      "WARNING:tensorflow:From <ipython-input-1-af68e9c263e1>:92: softmax_cross_entropy_with_logits (from tensorflow.python.ops.nn_ops) is deprecated and will be removed in a future version.\n",
      "Instructions for updating:\n",
      "\n",
      "Future major versions of TensorFlow will allow gradients to flow\n",
      "into the labels input on backprop by default.\n",
      "\n",
      "See tf.nn.softmax_cross_entropy_with_logits_v2.\n",
      "\n",
      "step: 100, entropy loss: 0.153759, l2_loss: 281.722992, total loss: 0.165028\n",
      "accuracy: 0.935400\n",
      "current learning rate: 0.010000\n",
      "step: 200, entropy loss: 0.292620, l2_loss: 393.526733, total loss: 0.308361\n",
      "accuracy: 0.945200\n",
      "current learning rate: 0.010000\n",
      "step: 300, entropy loss: 0.098599, l2_loss: 490.388550, total loss: 0.118214\n",
      "accuracy: 0.950200\n",
      "current learning rate: 0.010000\n",
      "step: 400, entropy loss: 0.083712, l2_loss: 588.984924, total loss: 0.107271\n",
      "accuracy: 0.951400\n",
      "current learning rate: 0.010000\n",
      "step: 500, entropy loss: 0.261659, l2_loss: 666.788635, total loss: 0.288330\n",
      "accuracy: 0.962300\n",
      "current learning rate: 0.010000\n",
      "step: 600, entropy loss: 0.098919, l2_loss: 734.440552, total loss: 0.128297\n",
      "accuracy: 0.959600\n",
      "current learning rate: 0.010000\n",
      "step: 700, entropy loss: 0.047022, l2_loss: 730.662842, total loss: 0.076249\n",
      "accuracy: 0.969300\n",
      "current learning rate: 0.005750\n",
      "step: 800, entropy loss: 0.134510, l2_loss: 722.145996, total loss: 0.163396\n",
      "accuracy: 0.971700\n",
      "current learning rate: 0.005750\n",
      "step: 900, entropy loss: 0.045158, l2_loss: 721.070740, total loss: 0.074001\n",
      "accuracy: 0.971700\n",
      "current learning rate: 0.005750\n",
      "step: 1000, entropy loss: 0.146745, l2_loss: 724.187683, total loss: 0.175712\n",
      "accuracy: 0.972100\n",
      "current learning rate: 0.005750\n",
      "step: 1100, entropy loss: 0.083840, l2_loss: 730.903687, total loss: 0.113076\n",
      "accuracy: 0.970300\n",
      "current learning rate: 0.005750\n",
      "step: 1200, entropy loss: 0.081016, l2_loss: 731.778442, total loss: 0.110287\n",
      "accuracy: 0.972200\n",
      "current learning rate: 0.005750\n",
      "step: 1300, entropy loss: 0.031867, l2_loss: 720.690979, total loss: 0.060695\n",
      "accuracy: 0.976800\n",
      "current learning rate: 0.003306\n",
      "step: 1400, entropy loss: 0.013570, l2_loss: 711.621582, total loss: 0.042035\n",
      "accuracy: 0.975700\n",
      "current learning rate: 0.003306\n",
      "step: 1500, entropy loss: 0.047936, l2_loss: 705.351929, total loss: 0.076150\n",
      "accuracy: 0.978700\n",
      "current learning rate: 0.003306\n",
      "step: 1600, entropy loss: 0.079516, l2_loss: 700.768738, total loss: 0.107547\n",
      "accuracy: 0.977300\n",
      "current learning rate: 0.003306\n",
      "step: 1700, entropy loss: 0.052144, l2_loss: 693.517517, total loss: 0.079885\n",
      "accuracy: 0.978800\n",
      "current learning rate: 0.003306\n",
      "step: 1800, entropy loss: 0.011399, l2_loss: 689.214722, total loss: 0.038968\n",
      "accuracy: 0.978100\n",
      "current learning rate: 0.003306\n",
      "step: 1900, entropy loss: 0.015655, l2_loss: 683.766785, total loss: 0.043006\n",
      "accuracy: 0.981300\n",
      "current learning rate: 0.001901\n",
      "step: 2000, entropy loss: 0.029102, l2_loss: 676.893188, total loss: 0.056178\n",
      "accuracy: 0.978300\n",
      "current learning rate: 0.001901\n",
      "step: 2100, entropy loss: 0.030594, l2_loss: 670.137695, total loss: 0.057400\n",
      "accuracy: 0.979600\n",
      "current learning rate: 0.001901\n",
      "step: 2200, entropy loss: 0.029687, l2_loss: 662.981812, total loss: 0.056206\n",
      "accuracy: 0.979500\n",
      "current learning rate: 0.001901\n",
      "step: 2300, entropy loss: 0.004308, l2_loss: 656.542786, total loss: 0.030570\n",
      "accuracy: 0.979500\n",
      "current learning rate: 0.001901\n",
      "step: 2400, entropy loss: 0.056663, l2_loss: 649.665894, total loss: 0.082650\n",
      "accuracy: 0.980900\n",
      "current learning rate: 0.001901\n",
      "step: 2500, entropy loss: 0.018058, l2_loss: 644.376160, total loss: 0.043833\n",
      "accuracy: 0.980500\n",
      "current learning rate: 0.001093\n",
      "step: 2600, entropy loss: 0.027299, l2_loss: 640.506531, total loss: 0.052920\n",
      "accuracy: 0.980700\n",
      "current learning rate: 0.001093\n",
      "step: 2700, entropy loss: 0.043601, l2_loss: 635.832153, total loss: 0.069034\n",
      "accuracy: 0.981100\n",
      "current learning rate: 0.001093\n",
      "step: 2800, entropy loss: 0.013564, l2_loss: 631.597778, total loss: 0.038828\n",
      "accuracy: 0.980700\n",
      "current learning rate: 0.001093\n",
      "step: 2900, entropy loss: 0.004050, l2_loss: 625.259399, total loss: 0.029060\n",
      "accuracy: 0.981300\n",
      "current learning rate: 0.001093\n",
      "step: 3000, entropy loss: 0.018622, l2_loss: 620.572510, total loss: 0.043444\n",
      "accuracy: 0.980800\n",
      "current learning rate: 0.001093\n"
     ]
    }
   ],
   "source": [
    "\"\"\"A very simple MNIST classifier.\n",
    "See extensive documentation at\n",
    "https://www.tensorflow.org/get_started/mnist/beginners\n",
    "\"\"\"\n",
    "from __future__ import absolute_import\n",
    "from __future__ import division\n",
    "from __future__ import print_function\n",
    "\n",
    "import argparse\n",
    "import sys\n",
    "import numpy as np\n",
    "\n",
    "from tensorflow.examples.tutorials.mnist import input_data\n",
    "\n",
    "import tensorflow as tf\n",
    "\n",
    "FLAGS = None\n",
    "\n",
    "def swish(x):\n",
    "  return x * tf.nn.sigmoid(x)\n",
    "\n",
    "\n",
    "def selu(x):\n",
    "  with tf.name_scope('elu') as scope:\n",
    "    alpha = 1.6732632423543772848170429916717\n",
    "    scale = 1.0507009873554804934193349852946\n",
    "    return scale*tf.where(x>=0.0, x, alpha*tf.nn.elu(x))\n",
    "\n",
    "def relu(x):\n",
    "    return tf.nn.relu(x)\n",
    "\n",
    "def activation(x):\n",
    "#  return selu(x)\n",
    "#  return relu(x)\n",
    "#  return tf.nn.sigmoid(x)\n",
    "#  return tf.nn.elu(x)\n",
    "  return swish(x)\n",
    "\n",
    "def initialize(shape, stddev=0.1):\n",
    "  return tf.truncated_normal(shape, stddev=stddev)\n",
    "  #return tf.zeros(shape)\n",
    "\n",
    "\n",
    "# Import data\n",
    "data_dir = '/home/jhon/data'\n",
    "mnist = input_data.read_data_sets(data_dir, one_hot=True)\n",
    "\n",
    "init_learning_rate = tf.placeholder(tf.float32)\n",
    "\n",
    "# Create the model\n",
    "L1_units_count = 100\n",
    "x = tf.placeholder(tf.float32, [None, 784])\n",
    "#tf.shape(x)  [100, 784]\n",
    "# exponetial lr decay\n",
    "epoch_steps = tf.to_int64(tf.div(60000, tf.shape(x)[0]))\n",
    "global_step = tf.train.get_or_create_global_step()\n",
    "current_epoch = global_step//epoch_steps\n",
    "decay_times = current_epoch \n",
    "current_learning_rate = tf.multiply(init_learning_rate, \n",
    "                                    tf.pow(0.575, tf.to_float(decay_times)))\n",
    "\n",
    "W_1 = tf.Variable(initialize([784, L1_units_count], \n",
    "                             stddev=np.sqrt(2/784)))\n",
    "b_1 = tf.Variable(tf.constant(0.001, shape=[L1_units_count]))\n",
    "logits_1 = tf.matmul(x, W_1) + b_1\n",
    "output_1 = activation(logits_1)\n",
    "\n",
    "L2_units_count = 10 \n",
    "W_2 = tf.Variable(initialize([L1_units_count, \n",
    "                              L2_units_count], \n",
    "                             stddev=np.sqrt(2/L1_units_count)))\n",
    "b_2 = tf.Variable(tf.constant(0.001, shape=[L2_units_count]))\n",
    "logits_2 = tf.matmul(output_1, W_2) + b_2  \n",
    "\n",
    "y = logits_2\n",
    "\n",
    "\n",
    "\n",
    "# Define loss and optimizer\n",
    "y_ = tf.placeholder(tf.float32, [None, 10])\n",
    "\n",
    "# The raw formulation of cross-entropy,\n",
    "#\n",
    "#   tf.reduce_mean(-tf.reduce_sum(y_ * tf.log(tf.nn.softmax(y)),\n",
    "#                                 reduction_indices=[1]))\n",
    "#\n",
    "# can be numerically unstable.\n",
    "#\n",
    "# So here we use tf.nn.softmax_cross_entropy_with_logits on the raw\n",
    "# outputs of 'y', and then average across the batch.\n",
    "cross_entropy = tf.reduce_mean(\n",
    "    tf.nn.softmax_cross_entropy_with_logits(labels=y_, logits=y))\n",
    "l2_loss = tf.nn.l2_loss(W_1) + tf.nn.l2_loss(W_2)\n",
    "total_loss = cross_entropy + 4e-5*l2_loss\n",
    "\n",
    "#optimizer = tf.train.AdamOptimizer(current_learning_rate)\n",
    "#gradients = optimizer.compute_gradients(total_loss)\n",
    "#train_step = optimizer.apply_gradients(gradients)\n",
    "\n",
    "train_step = tf.train.AdamOptimizer(\n",
    "    current_learning_rate).minimize(\n",
    "    total_loss, global_step=global_step)\n",
    "\n",
    "\n",
    "\n",
    "correct_prediction = tf.equal(tf.argmax(y, 1), tf.argmax(y_, 1))\n",
    "accuracy = tf.reduce_mean(tf.cast(correct_prediction, tf.float32))\n",
    "\n",
    "sess = tf.InteractiveSession()\n",
    "tf.global_variables_initializer().run()\n",
    "# Train\n",
    "for step in range(3000):\n",
    "  batch_xs, batch_ys = mnist.train.next_batch(100)\n",
    "  lr = 1e-2\n",
    "  _, loss, l2_loss_value, total_loss_value, current_lr_value = \\\n",
    "      sess.run(\n",
    "               [train_step, cross_entropy, l2_loss, total_loss, \n",
    "                current_learning_rate], \n",
    "               feed_dict={x: batch_xs, y_: batch_ys, \n",
    "                          init_learning_rate:lr})\n",
    "  \n",
    "  if (step+1) % 100 == 0:\n",
    "    print('step: %d, entropy loss: %f, l2_loss: %f, total loss: %f' % \n",
    "            (step+1, loss, l2_loss_value, total_loss_value))\n",
    "    #print(sess.run(accuracy, feed_dict={x: batch_xs, y_: batch_ys}))\n",
    "    print('accuracy: %f'%(sess.run(accuracy, feed_dict={x: mnist.test.images,\n",
    "                                    y_: mnist.test.labels})))\n",
    "    print('current learning rate: %f'%current_lr_value)\n",
    "    "
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": []
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.7"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
